package com.tom.service.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tom.util.HttpRequest;
import com.tom.util.JDBCUtil;

public class DZ_Qdsz {
	/**
	 * 
	 * @author hanpengda
	 * @date 2018年5月21日
	 * @TODO 获取青岛市政url
	 */
	public static void work() {
		try {
			ArrayList<String> urList = JDBCUtil.getUrls("qss_dz.qss_qdsz_urls");
			ArrayList<String> list = new ArrayList<String>();
			for (int i = 0; i < urList.size(); i++) {
				String url = urList.get(i);
				// System.out.println(url);
				String[] urls = url.split("###");
				for (int j = 0; j < urls.length; j++) {
					// System.out.println(urls[j]);
					url = urls[j];
					// System.out.println(url);
					Date now = new Date();
					SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
					String time = dateFormat.format(now);
					String sql = "insert into qss_dz.qss_qdsz_url(url,insertdate) values(" + "'" + url + "'" + "," + "'"
							+ time + "'" + ")";
					list.add(sql);
					System.out.println(sql);
				}
			}
			//添加数据库操作
			//JDBCUtil.insertData(list);
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}
	/**
	 * 
	 * @author hanpengda
	 * @date 2018年5月21日
	 * @TODO 爬取青岛市政数据
	 */
	public static void work2() {
		String params[] = null;
		String param = null;
		String result = null;
		// 进入数据网址之前发送的postUrl
		String postUrl = "http://govmail.qingdao.gov.cn/Government/ajaxpro/Yawei.Government.GovWebSite.GovReceiveQueAjax,Yawei.Government.ashx";
		// 抓取数据的网址
		String getUrl = "http://govmail.qingdao.gov.cn/government/GovWebSite/ReplyInfos.aspx?guid=";
		String html = null;
		// XPath解析规则
		String exp = "//div[@class='dafu_kuang']/table/tbody//tr[not(@style)]/td";
		Object resultData;
		ArrayList<String> sqlList = new ArrayList<String>(500);
		try {
			ArrayList<String> urList = JDBCUtil.getUrls("qss_dz.qss_qdsz_url");
			for (int i = 0; i < urList.size(); i++) {
				params = urList.get(i).split(",");
				param = "{\"queGuid\":\"" + params[0] + "\",\"isSecret\":\"" + params[1] + "\",\"seekcode\":\"\"}";
				result = sendPost(param, postUrl);
				JSONObject object = JSONObject.parseObject(result);
				if (Integer.valueOf(object.getString("value")) == 0) {
					//jsoup解析出网页源代码
					Connection connect = Jsoup.connect(getUrl + params[0]).cookie("Cookie",
							"yfx_c_g_u_id_10002431=_ck18052210390911340005243417690; yfx_mr_10002431=%3A%3Amarket_type_free_search%3A%3A%3A%3Abaidu%3A%3A%3A%3A%3A%3A%3A%3Awww.baidu.com%3A%3A%3A%3Apmf_from_free_search; yfx_mr_f_10002431=%3A%3Amarket_type_free_search%3A%3A%3A%3Abaidu%3A%3A%3A%3A%3A%3A%3A%3Awww.baidu.com%3A%3A%3A%3Apmf_from_free_search; yfx_key_10002431=; yfx_f_l_v_t_10002431=f_t_1526956749122__r_t_1527037969104__v_t_1527041308322__r_c_1; ASP.NET_SessionId=b1tsofaiiwjuwdjmz2pvcgbm");
					html = connect.get().body().html();
					//xpath解析所需内容
					HtmlCleaner hc = new HtmlCleaner();
					TagNode tn = hc.clean(html);
					Document dom = new DomSerializer(new CleanerProperties()).createDOM(tn);
					XPath xPath = XPathFactory.newInstance().newXPath();
					resultData = xPath.evaluate(exp, dom, XPathConstants.NODESET);
					if (resultData instanceof NodeList) {
						NodeList nodeList = (NodeList) resultData;
						String values = "";
						String sql = "insert into qss_dz.qss_dazong_qingdaoshizheng(letter_theme,letter_writer,send_time,letter_content,handling_process,reply_department,reply_time,replay_content,keyword,insertdate)";
						for (int j = 0; j < nodeList.getLength(); j ++) {
							//抓取内容
							if (nodeList.item(j).getTextContent().equals("信件主题 ")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("写 信 人 ")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("发信时间 ") || nodeList.item(j).getTextContent().equals("发信时间：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("信件内容 ") || nodeList.item(j).getTextContent().equals("信件内容：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("办理流程 ") || nodeList.item(j).getTextContent().equals("办理流程：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("答复部门 ") || nodeList.item(j).getTextContent().equals("答复部门：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("答复时间 ") || nodeList.item(j).getTextContent().equals("答复时间：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("答复信息 ") || nodeList.item(j).getTextContent().equals("答复信息：")) {
								values += "," + "'" + nodeList.item(j + 1).getTextContent().replaceAll("'", "") + "'";
							}
							if (nodeList.item(j).getTextContent().equals("第 2 次提问")) {
								sql = "insert into qss_dz.qss_dazong_qingdaoshizheng(letter_theme,letter_writer,send_time,letter_content,handling_process,"
										+ "reply_department,reply_time,replay_content,is_second_question,send_time_second,letter_content_second,handling_process_second,"
										+ "reply_department_second,reply_time_second,replay_content_second,keyword,insertdate)";
								values += "," + "'是'";
							}
							if (nodeList.item(j).getTextContent().equals("第 3 次提问")) {
								break;
							}
						}
						// 获取当前时间
						Date now = new Date();
						SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
						String time = dateFormat.format(now);
						// 拼接sql完成并保存到集合中
						values = values.substring(1) + ",'华银'" + "," + "'" + time + "'";
						sql += "values(" + values + ")";
						System.out.println(sql);
						sqlList.add(sql);
					}
				}
			}
			// 添加数据库操作
			//JDBCUtil.insertData(sqlList);
		} catch (Exception e) {
			e.printStackTrace();
		}

	}

	/**
	 * 
	 * @author hanpengda
	 * @date 2018年5月22日
	 * @TODO 发送post请求
	 */
	public static String sendPost(String sendData, String requestUrl) {
		StringBuffer jsonString;
		try {
			URL url = new URL(requestUrl);
			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
			connection.setDoInput(true);
			connection.setDoOutput(true);
			connection.setRequestMethod("POST");
			// 设置header
			connection.setRequestProperty("Accept", "*/*");
			connection.setRequestProperty("Content-Type", "text/plain; charset=UTF-8");
			connection.setRequestProperty("Cookie",
					"yfx_c_g_u_id_10002431=_ck18052210390911340005243417690; yfx_mr_10002431=%3A%3Amarket_type_free_search%3A%3A%3A%3Abaidu%3A%3A%3A%3A%3A%3A%3A%3Awww.baidu.com%3A%3A%3A%3Apmf_from_free_search; yfx_mr_f_10002431=%3A%3Amarket_type_free_search%3A%3A%3A%3Abaidu%3A%3A%3A%3A%3A%3A%3A%3Awww.baidu.com%3A%3A%3A%3Apmf_from_free_search; yfx_key_10002431=; yfx_f_l_v_t_10002431=f_t_1526956749122__r_t_1527037969104__v_t_1527041308322__r_c_1; ASP.NET_SessionId=b1tsofaiiwjuwdjmz2pvcgbm");
			connection.setRequestProperty("Host", "govmail.qingdao.gov.cn");
			connection.setRequestProperty("Referer",
					"http://govmail.qingdao.gov.cn/government/Mainfrm.aspx?keyword=%25E5%258D%258E%25E9%2593%25B6");
			connection.setRequestProperty("X-AjaxPro-Method", "IsCheckInfo");
			OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream(), "UTF-8");
			writer.write(sendData);
			writer.close();
			BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()));
			jsonString = new StringBuffer();
			String line;
			while ((line = br.readLine()) != null) {
				jsonString.append(line);
			}
			br.close();
			connection.disconnect();
		} catch (Exception e) {
			throw new RuntimeException(e.getMessage());
		}
		return jsonString.toString();
	}
}
